## Warning: package 'knitr' was built under R version 3.4.3
## ── Attaching packages ────────────────────────────────── tidyverse 1.2.1 ──
## ✔ ggplot2 2.2.1     ✔ purrr   0.2.5
## ✔ tibble  1.4.2     ✔ dplyr   0.7.5
## ✔ tidyr   0.8.1     ✔ stringr 1.3.1
## ✔ readr   1.1.1     ✔ forcats 0.3.0
## Warning: package 'tibble' was built under R version 3.4.3
## Warning: package 'tidyr' was built under R version 3.4.4
## Warning: package 'purrr' was built under R version 3.4.4
## Warning: package 'dplyr' was built under R version 3.4.4
## Warning: package 'stringr' was built under R version 3.4.4
## Warning: package 'forcats' was built under R version 3.4.3
## ── Conflicts ───────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## 
## Attaching package: 'assertthat'
## The following object is masked from 'package:tibble':
## 
##     has_name
## Warning: package 'lme4' was built under R version 3.4.4
## Loading required package: Matrix
## 
## Attaching package: 'Matrix'
## The following object is masked from 'package:tidyr':
## 
##     expand
## 
## Attaching package: 'langcog'
## The following objects are masked from 'package:ggthemes':
## 
##     scale_color_solarized, scale_colour_solarized,
##     scale_fill_solarized
## The following object is masked from 'package:base':
## 
##     scale
## Warning: package 'lmerTest' was built under R version 3.4.3
## 
## Attaching package: 'lmerTest'
## The following object is masked from 'package:lme4':
## 
##     lmer
## The following object is masked from 'package:stats':
## 
##     step
## Warning: package 'viridis' was built under R version 3.4.3
## Loading required package: viridisLite
## Warning: package 'jsonlite' was built under R version 3.4.4
## 
## Attaching package: 'jsonlite'
## The following object is masked from 'package:purrr':
## 
##     flatten
## Loading required package: gridExtra
## 
## Attaching package: 'gridExtra'
## The following object is masked from 'package:dplyr':
## 
##     combine

Part 1: Setup

Load in recognition data

d.raw <- data.frame()
files <- dir("raw_recognition_data/")
for (f in files) {
  jf <- paste("raw_recognition_data/",f,sep="")
  jd <- fromJSON(paste(readLines(jf), collapse=""))
  id <- data.frame(workerid = jd$WorkerId, 
                   guessed_category = jd$answers$data$rating,
                   actual_category = jd$answers$data$this_sketch_category,
                   subID = jd$answers$data$drawing_session_id,
                   condition = jd$answers$data$condition,
                   this_sketch_name = jd$answers$data$this_sketch_name,
                   age = jd$answers$data$producer_age,
                   photo_cue = jd$answers$data$photo_cue,
                   chunk = jd$answers$data$chunk,
                   trial_number =jd$answers$data$chunk
                  )
  d.raw <- bind_rows(d.raw, id)
}
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-32KTQ2V7RDFV5S57I2D3RZGWFFP9MB.json'
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-358010RM5ET4WBHEZAK8UYLRDBRXVY.json'
## Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3IAS3U3I0FGOMHGB648F3WXOXAB2BN.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3LKC68YZ3A3UHR3ROT28XNDC8HQOWY.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3PXX5PX6LXYDRKBFJJ1U9EGM9YEBAE.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3UWN2HHPUY5JSPM639SUL9AMCTNSN4.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3VHHR074H3H7LRZT3TE4DQA1KKGL7E.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 301KG0KX9CLVQAL3GUD90N5YHPTH26-3Z3ZLGNNSIU0B5EJTMJAHVXFPX6Q3A.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-31HQ4X3T3SAT4PJLRWCF8H6X7Z8LSI.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-39U1BHVTDLRD3L6BAQD5M0829WVT31.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3B3WTRP3DB25YO1E8KUJCQ2Y9FK926.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3CN4LGXD5XOU23O9KA5N4MSN2YZY46.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3D4CH1LGEATVDI9L7ZF8EUWZJ27G9M.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3IO1LGZLK9XT2K15QU0827OZ5AC86X.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3KYQYYSHYV7V8L307YXI5ZOR113DOX.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EN4YVUOUCSSIWJQSJ1N2QF97Q1JXR-3UXUOQ9OKEXQP88P62PE4CUQQAE7AJ.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-30H4UDGLT2IGXFLEZR90DPGBKJ6PML.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-33IZTU6J811O4J0A5FF3JB1KE2MSX3.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-34T446B1C0E00C535H0WVSARISG0CN.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-37Z929RLG98HN2RQ05E6U8GE16YSTK.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-39JEC7537U1GTD6J8EOLGL0HCOSVC9.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-3R2UR8A0IAG96R8HYYYUZ6RJHDJXOB.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-3UOUJI6MTDE4JWSEUKA3YCJ8ZDOXUX.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3EQVJH0T40LPA9FGZ9X51AEOH02TH2-3YHH42UU5BFT3GZ9IUTVYB0EC2I0LT.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-30X31N5D63QC86SHU5NPZ3QMDMOSAK.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-324G5B4FB38UOVA7E0SLF6IDHM007F.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-354P56DE9K3UP4UJN2B5PCOWCEIS7T.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-3AAJC4I4FGSKABHZJ3UAOVQ8QPKZJG.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-3I0BTBYZAXLD3F6819ILSJ2ALMEY0L.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-3KXIR214I4G41IV2RPYSVIRINJG427.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-3NVC2EB65QZ9K755KW04UH0HOXT3YE.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3HEADTGN2PSHCVX8099EGW4F1WKVRP-3TVRFO09GKF1065KL6I27PNPMDPXLF.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-30LB5CDZNCAD85GDXVK46QBCRFD0Z7.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-37ZHEEHM6WMQ5N94XNA4H4GKKZP37K.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-38F5OAUN5NC5ZVG3C8Z6LYHU35U7HK.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-38F71OA9GTW462WKLODV1V27IEIFM9.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-3LBXNTKX0RV6Z44BW9FC2GWUMJ99XV.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-3LRLIPTPEQ9EKNF9PWEN8HZCWUDAKR.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-3NXNZ5RS1AXCKPYKEWBE8HJ6KQU79Y.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3K8CQCU3KE19CM5LXCXTTEIJ78XWNC-3SNLUL3WO4N9J2BPG17FNIOLSYFULX.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-32N49TQG3GIS0Y9LO0SUAQATCQIVAS.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-336YQZE83VEC453VFC0MFJBUTF6M5P.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-38JBBYETQOAWWY7IK9TTXJ07KGIE4J.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-39OWYR0EPKR40JLY4R16VQPERIWYF6.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-3DZQRBDBSLFCOLTWLQ8FCVCG7CWS3O.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-3KAKFY4PGU2NU7Q0GEE18T6TYGUI3Q.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-3OB0CAO74HPOEFC4ZHP6YB1R7X9YH4.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3M7OI89LVYOSR3TOH4BBF9PUK0OC63-3VNXK88KKCIEVFZGWUEUEVZ9680V9D.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-32Q90QCQ1SLHA0WL51G940P3H79KEQ.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-39GAF6DQWR0W6AWLSHZ1ZPHQ8RBV1T.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3FFJ6VRIL1ORDFR3XRWI6V8MVX9I0G.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3R6BYFZZP7CF0EVOZVF4CCI5PH3XFA.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3TMSXRD2X609LZWQIR4XLR04F5F1WL.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3U0SRXB7CD57RAMTOKSPVZ5QQM3NR0.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3VBEN272MKZDIX5KG5FZVP1PBC0SGL.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ODOP6T3ASK1EYX81NTQ6YWUCI5421-3ZSANO2JCF774X9P5L9HD41DAX0SFM.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-32SCWG5HIH4E8C0MCB2JOTJPUJS6PT.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-382M9COHEHFVDW1XZFKBANY1J2HEUZ.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3CP1TO84PT1MX1Z25UH20AXWNEP25M.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3K5TEWLKGVB76GSJ0OK7ROYCP2XIV1.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3M68NM076H7ZKPG1P368TOJ4V1G6RL.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3S06PH7KSR4ACTWZ1WZ7T3BJ8TI1DT.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3U8YCDAGXPG4UDFMAZ1RFXZ8MCRQ02.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3QO7EE372ON2RGK2IDTEAU032I2QBD-3W8CV64QJ2Z9DE4WR5XQZU80YTQ9H2.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3BXQMRHWKZYTPKTAGD8UNV3AS5OUMM.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3DEL4X4EL6LL05CJ4L7S1XPLVSTXY7.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3EG49X351UCJHL2DY2C0I65SWI16XH.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3HHRAGRYX85J5VQJVISGJ351AF49OQ.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3L4D84MILZSYJQH5DHO3OOM3U2BHJM.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3OF2M9AATGOFLVNH1OUMPAQL5HSZKT.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3TAYZSBPLL8N33XDHQUATL7O1YZ2SQ.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3R5LWXWHR08IRUTPKDGK5DNTFYHGXH-3WZ36BJEV3GI6GAOPJSB4J6EXWTTBT.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-31QNSG6A5RTC6KFA94508LOV0KC789.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-33CKWXB73JKGKYGNL3FCS1AAJS611M.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-33TIN5LC04AVZ9UL151ELU33OP39Y1.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-3AQF3RZ558I2HZBSY86ECZ5LSD5F6Z.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-3BV8HQ2ZZW17L8UKNSKMYYUUV126AV.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-3DBQWDE4Y6YIMNOVRDFMSY0UY9FN5X.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-3VD82FOHKQOL3TXM7227LT69G07COA.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector
## Warning in readLines(jf): incomplete
## final line found on 'raw_recognition_data/
## 3ZZAYRN1I6RZ2Q133MSVLJBPTOKTOY-3X73LLYYQ1EU2G8QSFGVHV3K3J5HN9.json'
## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

## Warning in bind_rows_(x, .id): binding character and factor vector,
## coercing into character vector

Check number of raters per chunk and image

raters_per_chunk <- d.raw %>%
  distinct(workerid, chunk) %>%
  group_by(chunk) %>%
  summarize(count = n())

raters_per_image <- d.raw %>%
  distinct(this_sketch_name, workerid) %>%
  group_by(this_sketch_name) %>%
  summarize(count = n())

Make guess/actual cateogries comparable

d <- d.raw
d$guessed_category = as.factor(d$guessed_category)
d$actual_category <- factor(d$actual_category, levels=levels(d$guessed_category))

d <- d %>%
  mutate(correct_or_not = (actual_category == guessed_category)) 
## Warning: package 'bindrcpp' was built under R version 3.4.4

Compute bad workers (relaunch hits to replace in batches)

prac_thres = 11/12 ## 12 practice trials -- if you missed more than 1, exclude

# compute who didn't pass this threshold
bad_workers <- d %>%
  filter(condition=='practice') %>%
  group_by(workerid, chunk) %>%
  summarize(avg_prac_correct = mean(correct_or_not)) %>%
  filter(avg_prac_correct < prac_thres)

# filter from dataset
d <- d %>%
  filter(!workerid %in% bad_workers$workerid)

raters_per_chunk_filtered <- d %>%
  distinct(workerid, chunk) %>%
  group_by(chunk) %>%
  summarize(count = n())

Compute accuracy for each sketch, averaging over turkers

avg_by_sketch <- d %>%
  filter(condition != 'practice') %>%
  mutate(sketch_name = str_split_fixed(this_sketch_name, '/',9)[,9]) %>%
  mutate(sketch_name = as.factor(sketch_name)) %>%
  mutate(correct_or_not = (actual_category == guessed_category)) %>%
  group_by(sketch_name,age,condition,actual_category, subID) %>%
  summarize(avg_sketch_correct = mean(correct_or_not)) # averages over raters

Read in python descriptives and join with recognition ratings

sketch_descriptives <- read_csv('sketch_descriptives/Photodraw2_SRCD_cleaned_data_4-8.csv') %>%
  as.tibble() %>%
  mutate(actual_category = as.factor(category)) %>%
  mutate(subID = as.factor(subID)) %>%
  mutate(condition = as.factor(condition)) %>%
  mutate(age = as.factor(age_numeric)) %>%
  mutate(sketch_name = str_split_fixed(filename,'/',12)[,12]) %>%
  mutate(session_id = str_split_fixed(session_id, '_', 2)[,2]) %>%
  mutate(session_id = as.factor(session_id))
## Parsed with column specification:
## cols(
##   X = col_integer(),
##   session_id = col_character(),
##   trial_num = col_integer(),
##   category = col_character(),
##   age = col_character(),
##   submit_time = col_double(),
##   submit_date = col_datetime(format = ""),
##   num_strokes = col_integer(),
##   draw_duration_old = col_double(),
##   draw_duration_new = col_double(),
##   trial_duration = col_double(),
##   mean_intensity = col_double(),
##   bounding_box = col_character(),
##   filename = col_character(),
##   condition = col_character(),
##   CB = col_integer(),
##   subID = col_character(),
##   image_name = col_character(),
##   age_numeric = col_integer()
## )
joined <- left_join(avg_by_sketch,sketch_descriptives, by =c('sketch_name','subID','age','condition','actual_category'))
## Warning: Column `sketch_name` joining factor and character vector, coercing
## into character vector
## Warning: Column `subID` joining character vector and factor, coercing into
## character vector
## Warning: Column `age` joining factors with different levels, coercing to
## character vector
## Warning: Column `condition` joining factors with different levels, coercing
## to character vector
## Warning: Column `actual_category` joining factors with different levels,
## coercing to character vector

Load tracing data

## Load tracing data
tracing <- read.csv('tracing_scores/photodraw2_tracing_pred.csv') %>%
  mutate(age_numeric = age) %>%
  mutate(age = paste0("age",age_numeric)) %>%
  mutate(age = as.factor(age))

count_tracers <- read.csv('tracing_scores/photodraw2_tracing_pred.csv') %>%
  mutate(age_numeric = age) %>%
  mutate(age = paste0("age",age_numeric)) %>%
  mutate(age = as.factor(age)) %>%
  distinct(session_id,age) %>%
  group_by(age) %>%
  summarize(count = n())

Look at correlation between tracing scores for these two types of items

both_trials <- tracing %>%
  group_by(session_id) %>%
  mutate(count_tracings = n()) %>%
  filter(count_tracings > 1)

tracing_by_item <- both_trials %>%
  group_by(session_id) %>%
  summarize(square_rating = ord_max[category=='square'], shape_rating=ord_max[category=='shape'], age = age[1])

ggplot(tracing_by_item, aes(x=square_rating, y=shape_rating, col=age)) +
  geom_jitter(width=.2, height=.2,alpha=.8) +
  scale_color_viridis(discrete = TRUE) + 
  facet_wrap(~age)

tracing_reliability = cor.test(both_trials$ord_max[both_trials$category=="square"],both_trials$ord_max[both_trials$category=="shape"])

Correlation between tracing scores for shape vs. square across subjects = 0.45

## Extract relevant statistics
tracing_summary <- tracing %>%
  group_by(session_id) %>%
  # filter(category == "shape") %>%
  summarize(avg_tracing_rating = sum(ord_max)/n()) %>%# use instead of mean because sometimes that errors when we only have one tracing
  # mutate(avg_tracing_rating = scale(avg_tracing_rating)) %>%
  mutate(session_id = str_split_fixed(session_id, '_', 2)[,2])
  

## Join into one data frame
joined <- joined %>%
  mutate(session_id = as.factor(session_id)) %>%
  left_join(tracing_summary, by=c('session_id')) 
## Warning: Column `session_id` joining factor and character vector, coercing
## into character vector

Sanity check joining of two datasets

unique_sketches <- d %>%
  filter(condition!='practice') %>%
  distinct(this_sketch_name)
 
assert_that(length(unique_sketches$this_sketch_name)==length(joined$sketch_name))
## [1] TRUE

Part 2: Descriptive statistics

Take a look at the raw rating data

Highest chosen category is correct one; confusions look more or less reasonable.

ratingConfusions <- d %>%
  group_by(actual_category, guessed_category)  %>%
  summarize(number = n()) %>%
  group_by(actual_category) %>%
  mutate(prop = number / sum(number)) %>%
  complete(guessed_category, fill = list(prop = 0))

## Plot it
ggplot(ratingConfusions, 
       aes(x = guessed_category, y = actual_category, fill = prop)) + 
  geom_tile() + 
  ylab("True Category") + 
  xlab("Rated as") + 
  scale_fill_viridis(limits = c(0, .6),option="viridis") + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1, vjust = .5))

cor_by_indiv <- avg_by_sketch %>%
  group_by(subID) %>%
  summarize(mean_sub_correct = mean(avg_sketch_correct)) 
  
cor_by_indiv$subID = as.factor(cor_by_indiv$subID)
joined$subID = as.factor(joined$subID) 

out <- left_join(joined, cor_by_indiv) 
## Joining, by = "subID"
out$subID = fct_reorder(out$subID, out$mean_sub_correct, .desc=FALSE) ## fails in pipe?

ggplot(out, aes(x = subID, y = avg_sketch_correct, col = age)) +
  geom_jitter(width = .1, height=.1, alpha = .3) + 
  geom_point(aes(x=subID, y=mean_sub_correct, alpha=1)) +
  theme(axis.text.x = element_blank()) + 
  scale_color_viridis(discrete=TRUE) +
  facet_grid(~condition)

cor_by_indiv_by_cond <- avg_by_sketch %>%
  group_by(subID,condition) %>%
  summarize(mean_sub_correct = mean(avg_sketch_correct), age = age [1]) 

##
sub_reliability = cor.test(cor_by_indiv_by_cond$mean_sub_correct[cor_by_indiv_by_cond$condition=='P'],cor_by_indiv_by_cond$mean_sub_correct[cor_by_indiv_by_cond$condition=='S']) 
cor_by_indiv_by_cond_2 <- cor_by_indiv_by_cond %>%
  summarize(perception = mean_sub_correct[condition=='P'], memory = mean_sub_correct[condition=='S'], age = age[1])

ggplot(cor_by_indiv_by_cond_2, aes(x = memory, y = perception, col = age)) +
  geom_jitter(width = .1, height=.1, alpha = .8) + 
  theme_few(base_size = 16) +
  scale_color_viridis(discrete=TRUE) +
  # geom_smooth(alpha=.2) +
  geom_abline(slope=1, intercept=0, col='grey') +
  ylim(0,1) +
  xlim(0,1) + 
  xlab("Recognition Accuracy: Drawing from Memory") + 
  ylab("Recognition Accuracy: Drawing from Perception") 
## Warning: Removed 6 rows containing missing values (geom_point).

Correlation between ratings in P vs S condition across subjects = 0.75

Compute accuracy by condition/age

 cond_by_age <- avg_by_sketch %>%
  group_by(subID,condition,age) %>% # group at level of individual subjects/conditions
  summarize(avg_sub_by_cond_correct = mean(avg_sketch_correct)) %>% # average over sketch recognition accuracies
  group_by(age,condition) %>% # group by age/condition
  multi_boot_standard(col='avg_sub_by_cond_correct')  
not_practiced = c('hat','rabbit','watch','airplane','bike','chair')
practiced = c('fish','dog','bird','car','house','tree')

cond_by_age_prac <- avg_by_sketch %>%
  filter(actual_category %in% practiced ) %>%
  group_by(subID,condition,age) %>% # group at level of individual subjects/conditions
  summarize(avg_sub_by_cond_correct = mean(avg_sketch_correct)) %>% # average over sketch recognition accuracies
  group_by(age,condition) %>% # group by age/condition
  multi_boot_standard(col='avg_sub_by_cond_correct') 

 ggplot(cond_by_age_prac, aes(x=age, y=mean, col=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5))

joined <- joined %>%
  mutate(practiced_cat = (actual_category %in% practiced))
 
 
# model_lmer <- lmer(avg_sketch_correct ~ age_sc*condition*practiced_cat + avg_tracing_rating +  draw_duration_new + num_strokes + mean_intensity  + (condition | subID) + (condition | category),  data = joined)
# 
# model_summary = summary(model_lmer)$coef
# kable(model_summary)

Plot main results of recognition rtaings

ggplot(cond_by_age, aes(x=age, y=mean, col=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5))

Average and plot next to covariates

cor_by_age <- joined %>%
  group_by(subID,condition,age_numeric) %>%
  summarize(avg_cor = mean(avg_sketch_correct)) %>%
  group_by(age_numeric,condition) %>%
  multi_boot_standard(col = "avg_cor") 


joined$draw_duration_new[joined$draw_duration_new>30]=30 ## some weird draw durations > 30 seconds?
draw_duration <- joined %>%
  group_by(subID,condition,age_numeric) %>%
  summarize(avg_draw_duration = mean(draw_duration_new)) %>%
  group_by(age_numeric,condition) %>%
  multi_boot_standard(col = "avg_draw_duration")

num_strokes <- joined %>%
  group_by(subID,condition,age_numeric) %>%
  summarize(avg_num_strokes = mean(num_strokes)) %>%
  group_by(age_numeric,condition) %>%
  multi_boot_standard(col = "avg_num_strokes") 

avg_intensity <- joined %>%
  group_by(subID,condition,age_numeric) %>%
  summarize(avg_intensity = mean(mean_intensity)) %>%
  group_by(age_numeric,condition) %>%
  multi_boot_standard(col = "avg_intensity")

tracing_scores <- joined %>%
  group_by(subID, avg_tracing_rating) %>%
  distinct(subID, avg_tracing_rating,age_numeric) %>%
  filter(!is.na(avg_tracing_rating)) %>%
  group_by(age_numeric) %>%
  multi_boot_standard(col = "avg_tracing_rating")
tracing_visualize <- joined %>%
  group_by(subID, avg_tracing_rating) %>%
  distinct(subID, avg_tracing_rating,age_numeric) 

ggplot(tracing_visualize, aes(x=age_numeric, y=avg_tracing_rating, col=age_numeric)) +
  geom_jitter(width=.1, height=.1, alpha=.4) +
  scale_color_viridis()

Make compiled plot with accuracy and descriptives

## Make compiled plot of descriptives
base_size_chosen=18 # size of text in plots
smooth_alpha=.2

cor_by_age_plot_A = ggplot(cor_by_age, aes(age_numeric,mean, color=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5)) + 
  theme_few(base_size = base_size_chosen) + 
  labs(x='Age', y='Average adult recognition') +
  # scale_color_viridis(option="D") + 
  theme(legend.position = "none") +
  # geom_smooth(col='grey',span=10, alpha=smooth_alpha) +
  # ggtitle('A') + 
  ylim(0,1) + 
  geom_hline(yintercept = 1/49, linetype="dashed", color="grey")

p1=ggplot(draw_duration, aes(age_numeric,mean, color=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Age', y='Draw duration (s)') +
  # scale_color_viridis(option="D") +
  theme(legend.position = "none") + 
  ylim(0,20) 
  # geom_smooth(col='grey', span = 10) +
  # ggtitle('B')

p2=ggplot(avg_intensity, aes(age_numeric,mean, color=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Age', y='Ink used (mean intensity)') +
  # scale_color_viridis(option="D") +
  theme(legend.position = "none") + 
  ylim(.02,.08) 
  # geom_smooth(col='grey', span = 10,alpha=smooth_alpha)  +
  # ggtitle('C')

p3=ggplot(num_strokes, aes(age_numeric,mean, color=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Age', y='Number of strokes') +
  # scale_color_viridis(option="D") +
  theme(legend.position = "none") +
  ylim(0,15) 
  # geom_smooth(col='grey', span = 10,alpha=smooth_alpha)  +
  # ggtitle('D')
        
p4=ggplot(tracing_scores, aes(age_numeric,mean, color=age_numeric)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Age', y='Tracing score') +
  scale_color_viridis(option="D") +
  theme(legend.position = "none") +
  ylim(1,5) 
  # ggtitle('E')

Render jointed plot

ggarrange(cor_by_age_plot_A,p1,p2,p3,p4, nrow = 1)

## Compute target_label_prob averages within quantiles / age
num_quantiles = 5

# compute quantiles without knowing about age
joined$avg_tracing_rating_quantile = ntile(joined$avg_tracing_rating,num_quantiles)

# now compute averages
avg_by_tracing <- joined %>%
  filter(!is.na(avg_tracing_rating_quantile)) %>%
  group_by(avg_tracing_rating_quantile,age_numeric) %>%
  multi_boot_standard(col = "avg_sketch_correct")

avg_by_tracing$avg_tracing_rating_quantile = as.factor(avg_by_tracing$avg_tracing_rating_quantile)
# levels(avg_by_tracing$avg_tracing_rating_quantile)=c("Tracing Quartile 1","Tracing Quartile 2","Tracing Quartile 3","Tracing Quartile 4")

ggplot(avg_by_tracing, aes(age_numeric,mean, color=avg_tracing_rating_quantile)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper,alpha=.2)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Age', y='Proportion correct') +
  theme(legend.position = "none") +
  geom_smooth(col='grey', span = 10, method='lm')  +
  facet_grid(~avg_tracing_rating_quantile)

ggplot(avg_by_tracing, aes(avg_tracing_rating_quantile,mean, color=avg_tracing_rating_quantile)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper,alpha=.2)) +
  theme_few(base_size = base_size_chosen) +
  labs(x='Tracing Accuracy', y='Proportion correct') +
  theme(legend.position = "none") +
  geom_smooth(col='grey', span = 10, method='lm')  +
  facet_grid(~age_numeric) +
  scale_color_viridis(option='C', discrete=TRUE)

How different are individual participants?

## Get proportion recognized for each participant
indiv <- joined %>%
  group_by(subID) %>%
  mutate(sub_correct = mean(avg_sketch_correct)) %>%
  distinct(subID, sub_correct, age) %>% # get unique combos of sessionId and average 
  ungroup() %>%
  mutate(subID = fct_reorder(subID, sub_correct, .desc=TRUE))

## Plot individual children ordered from best to worst
ggplot(indiv, aes(x = subID, y = sub_correct, col = age)) +
  geom_point() +
  theme(axis.text.x = element_blank())+
  xlab("Individual children") +
  ylab("Proportion Recognized") +
  scale_color_viridis(discrete = TRUE) +
  geom_hline(yintercept=1/49, col='grey') # plot chance line

## Plot individual children by age
ggplot(indiv, aes(x = age, y = sub_correct, col = age)) +
  geom_point() +
  xlab("Age") +
  ylab("Proportion Recognized") +
  scale_color_viridis(discrete=TRUE) +
  geom_hline(yintercept=1/22, col='grey') # plot chance line

Look at individual children’s differences b/t perception vs. semantic across age

corbyChild <- joined %>%
  group_by(condition,subID, age) %>%
  summarize(avgCorrect = mean(avg_sketch_correct)) %>%
  group_by(subID,age) %>%
  summarize(cond_diff = avgCorrect[condition=='P'] - avgCorrect[condition=='S'])

ggplot(corbyChild, aes(x = age, y = cond_diff, col=age)) + 
  geom_jitter(width = .01, alpha = .5)  +
  scale_color_viridis(discrete=TRUE) +
  ylab("P-S")

Plot recognizability x num_strokes and drawing_duration

ggplot(joined, aes(x = num_strokes, y = avg_sketch_correct, col=condition)) +
    geom_jitter(width = .1, alpha = .5) +
    facet_grid(~age)

ggplot(joined, aes(x = draw_duration_new, y = avg_sketch_correct, col=condition)) +
    geom_jitter(width = .1, alpha = .5) +
    facet_grid(~age)

Part 3: Inferential Statistics

Inferential analyses: Non-linear mixed effect model with random slopes for items and random intercepts for participants.

# scale age, make sure condition is a factor
joined$condition=as.factor(joined$condition)

joined$age = as.numeric(joined$age)
joined$age_sc = scale(joined$age, scale = FALSE) # center but don't scale age

#
model_lmer <- lmer(avg_sketch_correct ~ age_sc*condition*avg_tracing_rating + (condition | subID) + (condition | category),  data = joined)

model_summary = summary(model_lmer)$coef
kable(model_summary)
Estimate Std. Error df t value Pr(>|t|)
(Intercept) 0.4899830 0.0515965 95.52974 9.4964352 0.0000000
age_sc 0.2016183 0.0319557 95.60532 6.3092997 0.0000000
conditionS -0.0794499 0.0520198 77.44412 -1.5273002 0.1307605
avg_tracing_rating 0.0446976 0.0181585 95.78595 2.4615236 0.0156236
age_sc:conditionS -0.0375113 0.0317499 95.24592 -1.1814618 0.2403609
age_sc:avg_tracing_rating -0.0325704 0.0122201 96.03899 -2.6653216 0.0090238
conditionS:avg_tracing_rating 0.0157274 0.0180657 95.50527 0.8705688 0.3861726
age_sc:conditionS:avg_tracing_rating 0.0086594 0.0121541 95.76096 0.7124692 0.4779067

Add effort covariates to see how this changes the model

#
model_lmer <- lmer(avg_sketch_correct ~ age_sc*condition*avg_tracing_rating + draw_duration_new + num_strokes + mean_intensity  + (condition | subID) + (condition | category),  data = joined)

model_summary = summary(model_lmer)$coef
kable(model_summary)
Estimate Std. Error df t value Pr(>|t|)
(Intercept) 0.4238995 0.0554421 133.57155 7.6458003 0.0000000
age_sc 0.1912288 0.0315324 96.55680 6.0645269 0.0000000
conditionS -0.0764784 0.0514227 72.20361 -1.4872500 0.1413030
avg_tracing_rating 0.0465854 0.0179354 97.08528 2.5974078 0.0108536
draw_duration_new -0.0009070 0.0018421 1007.60358 -0.4923979 0.6225453
num_strokes 0.0094932 0.0024924 1131.97399 3.8088298 0.0001471
mean_intensity -0.0237233 0.3782269 1123.92076 -0.0627223 0.9499988
age_sc:conditionS -0.0362762 0.0307172 93.69981 -1.1809757 0.2406014
age_sc:avg_tracing_rating -0.0307272 0.0120295 96.22718 -2.5543297 0.0122082
conditionS:avg_tracing_rating 0.0154898 0.0175178 94.66103 0.8842347 0.3788099
age_sc:conditionS:avg_tracing_rating 0.0073338 0.0117571 94.13211 0.6237743 0.5342845

Check residuals of the model

Seem relatively evenly distributed around zero; suggests lmer isn’t too bad of a fit – maybe should use glmer though

plot(joined$age,resid(model_lmer))

plot(joined$condition,resid(model_lmer))

Part 4: Compare human vs machine classification

Load classificaitons

classification_data <- read.csv('compiled_classifications/Classification_Outputs1200.csv') %>%
  as.tibble() %>%
  mutate(category = target_label) %>% 
  select(-X)

Join with human classifiation data (sizes will be different since some sketches eliminated for balancing by category)

all <- joined %>%
  left_join(classification_data)
## Joining, by = c("age", "session_id", "category")
## Warning: Column `session_id` joining character vector and factor, coercing
## into character vector
## Warning: Column `category` joining character vector and factor, coercing
## into character vector

Plot human classifications vs. machine classifications

ggplot(all, aes(x=avg_sketch_correct, y=correct_or_not, col=actual_category)) +
  geom_jitter(width=.01, height=.01, alpha=.5) +
  facet_wrap(~actual_category) + 
  geom_smooth() + 
  xlab('avg human accuracy') + 
  ylab('classification accuracy')
## `geom_smooth()` using method = 'loess'
## Warning: Removed 11 rows containing non-finite values (stat_smooth).
## Warning: Removed 11 rows containing missing values (geom_point).

human_advs <- all %>%
  filter(!is.na(correct_or_not)) %>%
  mutate(human_adv = avg_sketch_correct - correct_or_not) %>%
  group_by(age, actual_category) %>%
  multi_boot_standard(col='human_adv')

ggplot(human_advs, aes(x=age, y=mean, col=actual_category)) +
  geom_pointrange(aes(ymin = ci_lower, ymax=ci_upper)) +
  # geom_jitter(width=.01, height=.01, alpha=.5) +
  facet_wrap(~actual_category) +
  geom_hline(yintercept=0) + 
  ylab('human advantage')

Plot human classifications vs. machine target probability (graded measure of accuracy)

ggplot(all, aes(x=avg_sketch_correct, y=target_label_prob, col=actual_category)) +
  geom_jitter(width=.01, alpha=.5) +
  facet_wrap(~actual_category) + 
  geom_smooth() + 
  xlab('avg human accuracy') + 
  ylab('classification probability')
## `geom_smooth()` using method = 'loess'
## Warning: Removed 11 rows containing non-finite values (stat_smooth).
## Warning: Removed 11 rows containing missing values (geom_point).

Compare humans vs. machines for main contrast of interest

compare <- all %>%
  group_by(session_id,age,condition) %>%
  summarize(machine = mean(correct_or_not, na.rm=TRUE), human = mean(avg_sketch_correct)) 

compare <- gather(compare, value="accuracy", key="which_classifier", machine, human)
compare$which_classifier <- as.factor(compare$which_classifier)

age_by_cond <- compare %>%
  filter(!is.na(accuracy)) %>%
  group_by(condition,age,which_classifier) %>%
  multi_boot_standard(col='accuracy')

ggplot(age_by_cond, aes(x=age, y=mean, col=condition)) +
  geom_pointrange(aes(ymin = ci_lower, ymax = ci_upper), position = position_dodge(width = 0.5)) +
  facet_wrap(~which_classifier) +
  ylab('Recognition Accuracy') +
  xlab('Age')

Render out sketches where humans vs. machines had an advantage in recognitiontracing

human_advs_by_sketch <- all %>%
  filter(!is.na(correct_or_not)) %>%
  mutate(human_adv = avg_sketch_correct - correct_or_not) %>%
  filter(human_adv > quantile(human_adv, .75))

##
dir_name = 'human_advantage_drawings'
dir.create(dir_name)

out <- human_advs_by_sketch %>%
  mutate(image_path = paste('photodraw2_all_sketches/',category,'/',sketch_name,sep="")) %>%
  mutate(new_image_path = paste(dir_name,'/',round(human_adv,4),sketch_name,sep="")) 

file.copy(out$image_path, out$new_image_path)
machine_advs_by_sketch <- all %>%
  filter(!is.na(correct_or_not)) %>%
  mutate(machine_adv = correct_or_not - avg_sketch_correct) %>%
  filter(machine_adv > quantile(machine_adv, .75))


dir_name = 'machine_advantage_sketches'
dir.create(dir_name)

machine_out <- machine_advs_by_sketch %>%
  mutate(image_path = paste('photodraw2_all_sketches/',category,'/',sketch_name,sep="")) %>%
  mutate(new_image_path = paste(dir_name,'/',round(machine_adv,4),sketch_name,sep=""))

file.copy(machine_out$image_path, machine_out$new_image_path)

Render out the worst tracings in this dataset

scale_this <- function(x){
  (x - mean(x, na.rm=TRUE)) / sd(x, na.rm=TRUE)
}

bad_tracings <- tracing %>%
  mutate(tracing_score = ord_max) %>%
  filter(tracing_score < 3) 

dir_name = 'very_bad_tracings'
dir.create(dir_name)

tracing_out <- bad_tracings %>%
  mutate(image_path = paste('photodraw2_all_sketches/','/', 'this ',category,'/', 'S_this ',category,'_sketch_', age, '_', session_id,'.png',sep="")) %>%
  mutate(new_image_path = paste(dir_name,'/',round(tracing_score,4),'S_this ',category,'_sketch_', age, '_', session_id,'.png',sep=""))

file.copy(tracing_out$image_path, tracing_out$new_image_path)